package com.sl.crawler.robot.getUrl;

import com.sl.crawler.robot.CrawlerConfig;
import com.sl.crawler.robot.getUrl.selector.CssSelector;
import com.sl.crawler.robot.getUrl.selector.LinksSelector;
import com.sl.crawler.robot.request.Request;
import com.sl.crawler.robot.utils.UrlUtils;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Created by shiling on 2017/6/10.
 */
public class GetUrl {
    private final static Logger LOG = LoggerFactory.getLogger(GetUrl.class);
    private UrlQueue urlQueue = new UrlQueue();
    private String nextUrlCssPath;

    public void process(CrawlerConfig crawlerConfig, Request request, String html) {
        Document document = Jsoup.parse(html, request.getUrl());
        List<String> urlList = CssSelector.selectList(document, crawlerConfig.getNextUrlSelectPath(), "abs:href");
        LOG.info("新增需要遍历的url=" + urlList);
        addToUrlQueue(urlList, request);
    }

    /**
     * 添加到url队列
     *
     * @param urlList
     */
    public void addToUrlQueue(List<String> urlList) {
        for (String url : urlList) {
            urlQueue.push(new Request(url));
        }
    }

    /**
     * 添加到url队列
     *
     * @param request
     * @param urlList
     */
    public void addToUrlQueue(List<String> urlList, Request request) {
        for (String url : urlList) {
            if (StringUtils.isBlank(url) || StringUtils.equals(url, "#") || StringUtils.startsWithIgnoreCase(url, "javascript:")) {
                continue;
            }
            if (StringUtils.startsWithIgnoreCase(url, "http") && request != null) {
                url = UrlUtils.canonicalizeUrl(url, request.getUrl());
            }
            urlQueue.pushFirst(new Request(url));
        }
    }

    List<String> getHrefByRegex(String html, String regexPattern) {
        LinksSelector linksSelector = new LinksSelector();
        List<String> urlList = linksSelector.selectList(html);
        List<String> resultList = new ArrayList<>();
        Pattern pattern = Pattern.compile(regexPattern);
        for (String url : urlList) {
            Matcher matcher = pattern.matcher(url);
            if (matcher.find()) {
                if (matcher.groupCount() == 0) {
                    resultList.add(matcher.group(0));
                } else {
                    resultList.add(matcher.group(1));
                }
            }
        }

        return resultList;
    }

    public UrlQueue getUrlQueue() {
        return urlQueue;
    }

    public void setUrlQueue(UrlQueue urlQueue) {
        this.urlQueue = urlQueue;
    }


    public String getNextUrlCssPath() {
        return nextUrlCssPath;
    }

    public void setNextUrlCssPath(String nextUrlCssPath) {
        this.nextUrlCssPath = nextUrlCssPath;
    }

}
